{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-11-29T10:14:41.176742Z",
     "start_time": "2018-11-29T10:14:40.827501Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['[置顶]\\n步行街主干道版规（严禁人肉、时政敏感帖，禁发募捐帖，切勿轻信他人，谨防诈骗）', 'https://bbs.hupu.com/15806550.html', '虎扑团队', 'https://my.hupu.com/272380999276581', '2017-01-18', '1', '3228499', '虎扑团队', datetime.date(2017, 2, 27)]\n",
      "['抠图大师把底色换成蓝色，在车上公司要，没办法了', 'https://bbs.hupu.com/24555091.html', '沙棘汁', 'https://my.hupu.com/271275652956663', '2018-11-29', '1', '38', '把我的小熊', datetime.datetime(2018, 11, 29, 18, 14)]\n",
      "['#被父母取名禤靐龘# 香港一初中生崩溃：同学喊我“喂”ZT\\n\\n\\xa0\\n\\n[\\xa0\\n2\\n3...7\\n\\n\\xa0]', 'https://bbs.hupu.com/24554564.html', '晨灌吸', 'https://my.hupu.com/36713740238125', '2018-11-29', '122', '46656', 'o超级英雄o', datetime.datetime(2018, 11, 29, 18, 14)]\n",
      "['男的真的很在意腿吗？\\n\\n\\xa0\\n\\n[\\xa0\\n2\\n3...22\\n\\n\\xa0]', 'https://bbs.hupu.com/24554405.html', 'teihionn', 'https://my.hupu.com/170564828886404', '2018-11-29', '433', '119051', '天青色等烟俣丶', datetime.datetime(2018, 11, 29, 18, 14)]\n",
      "['作为外地人，对青岛和济南是怎么看的呢？\\n[\\xa0\\n2\\n\\n\\xa0]', 'https://bbs.hupu.com/24552866.html', '20181015', 'https://my.hupu.com/244388484603367', '2018-11-29', '22', '1017', '麦登登', datetime.datetime(2018, 11, 29, 18, 14)]\n",
      "['zt.调查：日本4成已婚男有婚外情经历，女星深田恭子成出轨理想对象\\n\\n\\xa0\\n\\n[\\xa0\\n2\\n3...13\\n\\n\\xa0]', 'https://bbs.hupu.com/24543703.html', '五行多二', 'https://my.hupu.com/234452147725348', '2018-11-28', '247', '239939', '亡灵勇士赛大脚', datetime.datetime(2018, 11, 29, 18, 14)]\n",
      "['刚在肯德基点了一份鸡块不小心全掉地上，我应该算了还是再点一份才是止损？', 'https://bbs.hupu.com/24554902.html', '秋名山顶老司机', 'https://my.hupu.com/17233445026571', '2018-11-29', '5', '389', 'wyx191', datetime.datetime(2018, 11, 29, 18, 14)]\n",
      "['媳妇做品牌代购，我跟着忙太太，怎样劝她\\n\\n\\xa0\\n\\n[\\xa0\\n2\\n\\n\\xa0]', 'https://bbs.hupu.com/24554462.html', 'CD', 'https://my.hupu.com/220671817495287', '2018-11-29', '28', '4264', '迷于归途', datetime.datetime(2018, 11, 29, 18, 14)]\n",
      "['小哥哥们，你们能不能接受去女方家过年三十？\\n\\n\\xa0\\n\\n[\\xa0\\n2\\n3...16\\n\\n\\xa0]', 'https://bbs.hupu.com/24554451.html', '侯晓瑾', 'https://my.hupu.com/194889165799246', '2018-11-29', '304', '44873', '我该取个什么名呢', datetime.datetime(2018, 11, 29, 18, 14)]\n",
      "['你睡觉的时候需要东西来辅助入睡吗？\\n\\n\\xa0\\n\\n[\\xa0\\n2\\n3...74\\n\\n\\xa0]', 'https://bbs.hupu.com/24548811.html', 'KINGLBJ第四冠', 'https://my.hupu.com/39068322831424', '2018-11-29', '1461', '352664', '梦想家陈大狗', datetime.datetime(2018, 11, 29, 18, 14)]\n",
      "['鉴定我家这柴犬柴不柴？\\n\\n\\xa0\\n\\n[\\xa0\\n2\\n3...15\\n\\n\\xa0]', 'https://bbs.hupu.com/24554073.html', 'cao3126415', 'https://my.hupu.com/122184812392841', '2018-11-29', '285', '125298', '泥巴妥妥夫斯基', datetime.datetime(2018, 11, 29, 18, 14)]\n",
      "['能认出10个以上的就是影迷，15个以上就是铁杆影迷\\n\\n\\xa0\\n\\n[\\xa0\\n2\\n3\\n4\\n\\n\\xa0]', 'https://bbs.hupu.com/24554013.html', '大连理工之魂', 'https://my.hupu.com/106500484775554', '2018-11-29', '78', '9212', '背锅侠阿扎', datetime.datetime(2018, 11, 29, 18, 14)]\n",
      "['相亲的妹子让我给她闺蜜介绍对象，精彩攻防令人窒息……\\n\\n\\xa0\\n\\n[\\xa0\\n2\\n3...11\\n\\n\\xa0]', 'https://bbs.hupu.com/24552416.html', '漸行漸遠', 'https://my.hupu.com/209307995003647', '2018-11-29', '203', '65499', '阿瓜爱篮球', datetime.datetime(2018, 11, 29, 18, 14)]\n",
      "['朋友圈这个女生，什么水平？\\n\\n\\xa0\\n\\n[\\xa0\\n2\\n3...7\\n\\n\\xa0]', 'https://bbs.hupu.com/24549509.html', '奶茶不如耐X', 'https://my.hupu.com/48238378389614', '2018-11-29', '134', '213657', '小丸子shy', datetime.datetime(2018, 11, 29, 18, 14)]\n",
      "['讨论一下关于学校管制发型的问题', 'https://bbs.hupu.com/24555037.html', '骚骚骚骚桓', 'https://my.hupu.com/122928183652228', '2018-11-29', '7', '66', '骚骚骚骚桓', datetime.datetime(2018, 11, 29, 18, 14)]\n",
      "['住房公积金能换城市吗？', 'https://bbs.hupu.com/24554790.html', '夜店熊', 'https://my.hupu.com/52102623726957', '2018-11-29', '13', '2223', '陈洋葱', datetime.datetime(2018, 11, 29, 18, 14)]\n",
      "['白百何陈羽凡跑男上演塑料夫妻情，被邓超吐槽 zt', 'https://bbs.hupu.com/24555097.html', '娱乐八卦阵', 'https://my.hupu.com/124819225349882', '2018-11-29', '2', '179', '菠萝草莓橙子', datetime.datetime(2018, 11, 29, 18, 14)]\n",
      "['蒋劲夫曾经因为打架被学校勒令退学,原来夫仔的拳头打小就硬\\n\\n\\xa0\\n\\n[\\xa0\\n2\\n3...6\\n\\n\\xa0]', 'https://bbs.hupu.com/24553466.html', '在小车', 'https://my.hupu.com/226136262605376', '2018-11-29', '101', '40507', 'LLLLLSQ', datetime.datetime(2018, 11, 29, 18, 14)]\n",
      "['张柏芝产下第三胎？工作人员一问三不知 zt\\n\\n\\xa0\\n\\n[\\xa0\\n2\\n3\\n\\n\\xa0]', 'https://bbs.hupu.com/24554778.html', '娱乐八卦阵', 'https://my.hupu.com/124819225349882', '2018-11-29', '54', '33233', '似梦Perseus', datetime.datetime(2018, 11, 29, 18, 14)]\n",
      "['罗斯，你从山贼变成了3贼，看到你的表现真的很开心啊', 'https://bbs.hupu.com/24555127.html', '步行街街道办事处经理', 'https://my.hupu.com/230787262361269', '2018-11-29', '1', '4', '步行街街道办事处经理', datetime.datetime(2018, 11, 29, 18, 14)]\n",
      "['大舅子出交通事故了', 'https://bbs.hupu.com/24555061.html', '克村带头大哥', 'https://my.hupu.com/124792729101697', '2018-11-29', '5', '327', 'LinOliver', datetime.datetime(2018, 11, 29, 18, 14)]\n",
      "['zt：FIFA最新世界排名：比利时第一，国足亚洲第7全球第76\\n[\\xa0\\n2\\n\\n\\xa0]', 'https://bbs.hupu.com/24554830.html', 'ysq我爱你', 'https://my.hupu.com/178133003440029', '2018-11-29', '26', '1645', '我是鸡腿侠', datetime.datetime(2018, 11, 29, 18, 14)]\n",
      "['1945年广岛投放原子弹全过程，全世界顷刻化为乌有\\n\\n\\xa0\\n\\n[\\xa0\\n2\\n3...22\\n\\n\\xa0]', 'https://bbs.hupu.com/24553765.html', '奔波霸儿', 'https://my.hupu.com/4040919041654', '2018-11-29', '434', '117366', '忍不住的时候会说', datetime.datetime(2018, 11, 29, 18, 14)]\n",
      "['3unshine组合Cindy登杂志 透视蕾丝装紫唇黑发笑容自信 zt\\n\\n\\xa0\\n\\n[\\xa0\\n2\\n3...21\\n\\n\\xa0]', 'https://bbs.hupu.com/24552775.html', '娱乐八卦阵', 'https://my.hupu.com/124819225349882', '2018-11-29', '418', '161762', '加一个吧我不会打', datetime.datetime(2018, 11, 29, 18, 14)]\n",
      "['我们学校的说唱什么水平', 'https://bbs.hupu.com/24554991.html', '乌蝇哥大佬', 'https://my.hupu.com/257516857496424', '2018-11-29', '4', '244', '乌蝇哥大佬', datetime.datetime(2018, 11, 29, 18, 14)]\n",
      "['讲真，野球场，这大哥一定会侮辱好多人的球商\\n\\n\\xa0\\n\\n[\\xa0\\n2\\n3...10\\n\\n\\xa0]', 'https://bbs.hupu.com/24554497.html', '长春的夜灯火阑珊', 'https://my.hupu.com/76117133384979', '2018-11-29', '193', '62236', '勒布朗何', datetime.datetime(2018, 11, 29, 18, 14)]\n",
      "['[爆照区]没有腿照，但是爱笑的小姐姐打分应该不会……太低吧……\\n\\n\\xa0\\n\\n[\\xa0\\n2\\n3...55\\n\\n\\xa0]', 'https://bbs.hupu.com/selfie', 'puppy是个瓜皮', 'https://my.hupu.com/221340416149417', '2018-11-29', '1080', '294197', '我韦还能再战', datetime.datetime(2018, 11, 29, 18, 14)]\n",
      "['一看就是个武林老前辈', 'https://bbs.hupu.com/24554950.html', 'Jrtwhite', 'https://my.hupu.com/278541205289189', '2018-11-29', '11', '1096', '嘻哈射什么', datetime.datetime(2018, 11, 29, 18, 14)]\n",
      "['同位置，平方，房间的情况下，这种应该是最完美的户型了吧？', 'https://bbs.hupu.com/24555132.html', 'Mu_露', 'https://my.hupu.com/204890305555492', '2018-11-29', '1', '3', 'Mu_露', datetime.datetime(2018, 11, 29, 18, 14)]\n",
      "['求助各位jr 苹果支付莫名其妙被扣138', 'https://bbs.hupu.com/24555086.html', '唐牛才是食神', 'https://my.hupu.com/51167744768604', '2018-11-29', '2', '81', 'aStepAhead', datetime.datetime(2018, 11, 29, 18, 14)]\n",
      "['【Eminem醒目】虎扑大神多，我这波Venom翻唱算什么水平？\\n\\n\\xa0\\n\\n[\\xa0\\n2\\n3...9\\n\\n\\xa0]', 'https://bbs.hupu.com/24554264.html', '火箭班卡卡罗特', 'https://my.hupu.com/106314943100531', '2018-11-29', '173', '17827', '火箭班卡卡罗特', datetime.datetime(2018, 11, 29, 18, 14)]\n",
      "['Wednesday / 自制飞机杯大全  ZT\\n\\n\\xa0\\n\\n[\\xa0\\n2\\n3\\n\\n\\xa0]', 'https://bbs.hupu.com/24554679.html', '痞子王丶', 'https://my.hupu.com/112073112956762', '2018-11-29', '54', '16403', '带不走忧伤', datetime.datetime(2018, 11, 29, 18, 14)]\n",
      "['李小璐贾乃亮同台尬唱《全部都是你》，全程眼神无交流，什么水平', 'https://bbs.hupu.com/24554851.html', 'Zza2013', 'https://my.hupu.com/188452316887420', '2018-11-29', '4', '524', '黄美英1019', datetime.datetime(2018, 11, 29, 18, 14)]\n",
      "['zt高铁十年:福建成为第一个市市通高铁的省份!\\n\\n\\xa0\\n\\n[\\xa0\\n2\\n3...43\\n\\n\\xa0]', 'https://bbs.hupu.com/24549009.html', '周靖雯真好看', 'https://my.hupu.com/207801063659695', '2018-11-29', '856', '225090', '化蝶肩上落', datetime.datetime(2018, 11, 29, 18, 14)]\n",
      "['腰疼胯疼好久，担心是强直，好害怕', 'https://bbs.hupu.com/24534928.html', '那个是甜瓜', 'https://my.hupu.com/32448647535305', '2018-11-27', '10', '928', '就是在上', datetime.datetime(2018, 11, 29, 18, 14)]\n",
      "['刚刚揪女朋友脖领子问话，现在要凉了。。\\n\\n\\xa0\\n\\n[\\xa0\\n2\\n3...8\\n\\n\\xa0]', 'https://bbs.hupu.com/24552606.html', 'HailCleve', 'https://my.hupu.com/145301199989922', '2018-11-29', '154', '110932', '城北徐公本人', datetime.datetime(2018, 11, 29, 18, 14)]\n",
      "['[什么图都有]什么图让你感叹这破街什么图都有？分享出来吧（11.28-12.4）\\n\\n\\xa0\\n\\n[\\xa0\\n2\\n3...39\\n\\n\\xa0]', 'https://bbs.hupu.com/4846', '菠萝夫司机', 'https://my.hupu.com/33336114263861', '2018-11-29', '770', '247259', '迷失于家埠', datetime.datetime(2018, 11, 29, 18, 14)]\n",
      "['JRs都来说说，虎扑在你心中是什么形象？你和虎扑有何故事？\\n\\n\\xa0\\n\\n[\\xa0\\n2\\n3...11\\n\\n\\xa0]', 'https://bbs.hupu.com/24554351.html', '南广王祖蓝', 'https://my.hupu.com/110407336160956', '2018-11-29', '203', '21485', '忠热篮球钟爱玫瑰', datetime.datetime(2018, 11, 29, 18, 14)]\n",
      "['树懒的一天是这样度过的！ zt', 'https://bbs.hupu.com/24551771.html', '萌宠的日常', 'https://my.hupu.com/83699750327603', '2018-11-29', '2', '472', 'lodger00', datetime.datetime(2018, 11, 29, 18, 14)]\n",
      "['[爆照区]今天你穿的什么鞋！\\n\\n\\xa0\\n\\n[\\xa0\\n2\\n3...128\\n\\n\\xa0]', 'https://bbs.hupu.com/selfie', 'hahawjm', 'https://my.hupu.com/59638649719977', '2018-11-29', '2551', '396336', '因为艾所以爱43', datetime.datetime(2018, 11, 29, 18, 14)]\n",
      "['李咏女儿新书面世 称在爸爸鼓励下创作小说 zt', 'https://bbs.hupu.com/24554895.html', '娱乐八卦阵', 'https://my.hupu.com/124819225349882', '2018-11-29', '20', '7031', '乃萌陌尔', datetime.datetime(2018, 11, 29, 18, 14)]\n",
      "['这两天打开虎扑全是秀腿，那么这种类型的当媳妇Jrs愿意吗zt', 'https://bbs.hupu.com/24555081.html', '老流氓队', 'https://my.hupu.com/209026743578452', '2018-11-29', '3', '132', '盘腿睡', datetime.datetime(2018, 11, 29, 18, 14)]\n",
      "['老妈珍藏的钱，连号的能卖多少钱\\n\\n\\xa0\\n\\n[\\xa0\\n2\\n3...19\\n\\n\\xa0]', 'https://bbs.hupu.com/24546718.html', '很困睡不着', 'https://my.hupu.com/216356651410956', '2018-11-28', '364', '412177', '暴扣弗拉哈', datetime.datetime(2018, 11, 29, 18, 14)]\n",
      "['这是高手！马琳大战王励勤，这球什么水平！', 'https://bbs.hupu.com/24555087.html', '绽放的三叶草', 'https://my.hupu.com/157890861356458', '2018-11-29', '1', '143', 'hongjin1111', datetime.datetime(2018, 11, 29, 18, 14)]\n",
      "['点播来了！羽泉在歌手演唱的狂流，这高音什么水平？\\n\\n\\xa0\\n\\n[\\xa0\\n2\\n3...12\\n\\n\\xa0]', 'https://bbs.hupu.com/24552574.html', '热火觉皇', 'https://my.hupu.com/204646073615892', '2018-11-29', '238', '84566', 'for利物浦', datetime.datetime(2018, 11, 29, 18, 14)]\n",
      "['QQ和微信一毛一样的功能，甚至更全面一些，为啥大家都放弃使用QQ转战微信\\n\\n\\xa0\\n\\n[\\xa0\\n2\\n3\\n\\n\\xa0]', 'https://bbs.hupu.com/24554443.html', '胡赞可', 'https://my.hupu.com/276425781398517', '2018-11-29', '57', '7266', '三月的跫音', datetime.datetime(2018, 11, 29, 18, 14)]\n",
      "['中国联通员工谈一下携号转网政策\\n\\n\\xa0\\n\\n[\\xa0\\n2\\n3...10\\n\\n\\xa0]', 'https://bbs.hupu.com/24554028.html', '西安邮电大学', 'https://my.hupu.com/229999978327495', '2018-11-29', '195', '30486', '拉瓦鲍尔', datetime.datetime(2018, 11, 29, 18, 14)]\n",
      "['晒出你2018年最难忘的一张图\\n\\n\\xa0\\n\\n[\\xa0\\n2\\n3...7\\n\\n\\xa0]', 'https://bbs.hupu.com/24553929.html', '老徐侃球', 'https://my.hupu.com/132225419087847', '2018-11-29', '125', '14499', '真是帅到没朋友', datetime.datetime(2018, 11, 29, 18, 14)]\n",
      "['[爆照区]女朋友问她在虎扑能打几分。。。\\n\\n\\xa0\\n\\n[\\xa0\\n2\\n3...24\\n\\n\\xa0]', 'https://bbs.hupu.com/selfie', '风骚怪物就是我', 'https://my.hupu.com/43359801623921', '2018-11-29', '474', '359846', '东方梁哥', datetime.datetime(2018, 11, 29, 18, 14)]\n",
      "['杨逍是否被低估？真实水平是什么？', 'https://bbs.hupu.com/24555090.html', '星宿老仙v法力无边', 'https://my.hupu.com/80983457581520', '2018-11-29', '2', '46', '星宿老仙v法力无边', datetime.datetime(2018, 11, 29, 18, 14)]\n",
      "['全国人大官网关于GWY增加加班补助，征求意见\\n\\n\\xa0\\n\\n[\\xa0\\n2\\n3...6\\n\\n\\xa0]', 'https://bbs.hupu.com/24552750.html', '一粒彪炳的小米饭', 'https://my.hupu.com/281316593454085', '2018-11-29', '112', '27141', '杰森斯坦张', datetime.datetime(2018, 11, 29, 18, 14)]\n",
      "['我这上篮能爆百分之一的jrs？各位28大的评评^_^\\n\\n\\xa0\\n\\n[\\xa0\\n2\\n3...10\\n\\n\\xa0]', 'https://bbs.hupu.com/24545512.html', '德鲁大叔jjj', 'https://my.hupu.com/15777156001504', '2018-11-28', '194', '34881', '靓仔鸽', datetime.datetime(2018, 11, 29, 18, 14)]\n",
      "['家人们，钱被诈骗之后还能要回来吗？', 'https://bbs.hupu.com/24555106.html', '我只爱圆圆', 'https://my.hupu.com/92890471082032', '2018-11-29', '1', '10', 'NBA资深记者', datetime.datetime(2018, 11, 29, 18, 14)]\n",
      "['晚上进小区门给钱到底合不合理？\\n\\n\\xa0\\n\\n[\\xa0\\n2\\n3...31\\n\\n\\xa0]', 'https://bbs.hupu.com/24547867.html', '七匹狼追梦', 'https://my.hupu.com/278033410836189', '2018-11-29', '610', '267916', '一百度贴吧官方间谍一', datetime.datetime(2018, 11, 29, 18, 14)]\n",
      "['刚加完班，想建一个专业的社保方面问题回复贴。\\n[\\xa0\\n2\\n3...6\\n\\n\\xa0]', 'https://bbs.hupu.com/24547328.html', '丹棱犟拐拐', 'https://my.hupu.com/13538474580777', '2018-11-28', '113', '4683', '吴wzhy', datetime.datetime(2018, 11, 29, 18, 14)]\n",
      "['有大神吗 求助p图高手', 'https://bbs.hupu.com/24554937.html', '我要帮楼主生孩子', 'https://my.hupu.com/64706096690118', '2018-11-29', '7', '139', 'LeMamba', datetime.datetime(2018, 11, 29, 18, 14)]\n",
      "['[数码区]王自如新iphone测评\\n\\n\\xa0\\n\\n[\\xa0\\n2\\n3...16\\n\\n\\xa0]', 'https://bbs.hupu.com/digital', '总想往高飞', 'https://my.hupu.com/159229048168202', '2018-11-29', '316', '146061', 'ywok8866', datetime.datetime(2018, 11, 29, 18, 14)]\n",
      "['马云1995年神预言：互联网改变中国 zt', 'https://bbs.hupu.com/24555089.html', '狂热数码控', 'https://my.hupu.com/87094574074268', '2018-11-29', '4', '169', 'wutthefuck', datetime.datetime(2018, 11, 29, 18, 14)]\n",
      "['[什么图都有]诱人！吃鸡女版小丑皮肤，蕾丝+项圈+双马尾！\\n\\n\\xa0\\n\\n[\\xa0\\n2\\n\\n\\xa0]', 'https://bbs.hupu.com/4846', '全县第一高中生', 'https://my.hupu.com/134513069308192', '2018-11-29', '24', '34745', 'Answer丿丶', datetime.datetime(2018, 11, 29, 18, 14)]\n",
      "['大家都来说说2018这一年赚了多少钱？', 'https://bbs.hupu.com/24555135.html', 'henrysc23', 'https://my.hupu.com/245932905504829', '2018-11-29', '0', '3', 'henrysc23', datetime.datetime(2018, 11, 29, 18, 13)]\n",
      "['韶关市长杯压哨绝杀什么水平？带上福利\\n\\n\\xa0\\n\\n[\\xa0\\n2\\n3\\n4\\n\\n\\xa0]', 'https://bbs.hupu.com/24554806.html', 'Cl-09', 'https://my.hupu.com/54886875560579', '2018-11-29', '76', '31763', '克克流浪', datetime.datetime(2018, 11, 29, 18, 13)]\n",
      "['吐槽大会还真是有毒。', 'https://bbs.hupu.com/24555134.html', '把球给我该回家了', 'https://my.hupu.com/248195244020237', '2018-11-29', '0', '2', '把球给我该回家了', datetime.datetime(2018, 11, 29, 18, 13)]\n",
      "['我在淘宝买到了一条假裤子', 'https://bbs.hupu.com/24554760.html', '龟龟龟龟威斯', 'https://my.hupu.com/78299421373267', '2018-11-29', '20', '1924', '龟龟龟龟威斯', datetime.datetime(2018, 11, 29, 18, 13)]\n",
      "['我这位同事的买车观什么水平？\\n\\n\\xa0\\n\\n[\\xa0\\n2\\n3...22\\n\\n\\xa0]', 'https://bbs.hupu.com/24549330.html', '天桥医院王医生', 'https://my.hupu.com/192701031224231', '2018-11-29', '433', '313598', '都是笑话', datetime.datetime(2018, 11, 29, 18, 13)]\n",
      "['[路人王专区]【用篮球表达爱】确认过眼神，是我想要的那个人。这波擦肩而过的爱意让人猝不及防\\n\\n\\xa0\\n\\n[\\xa0\\n2\\n3\\n\\n\\xa0]', 'https://bbs.hupu.com/lurenwang', '布布卡卡', 'https://my.hupu.com/123111929094203', '2018-11-29', '55', '25304', '想养一只狗', datetime.datetime(2018, 11, 29, 18, 13)]\n",
      "['你闻过最好闻的味道是什么？zt\\n\\n\\xa0\\n\\n[\\xa0\\n2\\n3...89\\n\\n\\xa0]', 'https://bbs.hupu.com/24551619.html', '取个名咋就这么难1', 'https://my.hupu.com/139563248353455', '2018-11-29', '1773', '476265', 'Citywalk', datetime.datetime(2018, 11, 29, 18, 13)]\n",
      "['如果戒烟难度是两颗星～那戒毒是难度是几颗星？\\n\\n\\xa0\\n\\n[\\xa0\\n2\\n3...10\\n\\n\\xa0]', 'https://bbs.hupu.com/24553832.html', '一夜五次郞丶', 'https://my.hupu.com/166846510374398', '2018-11-29', '198', '49287', 'Jenova_Zack', datetime.datetime(2018, 11, 29, 18, 13)]\n",
      "['公司强制要求10.5个工时', 'https://bbs.hupu.com/24555133.html', '锅哥丶', 'https://my.hupu.com/49550655384183', '2018-11-29', '0', '6', '锅哥丶', datetime.datetime(2018, 11, 29, 18, 13)]\n",
      "['用一个字概括你的2018，你会用哪个字？\\n\\n\\xa0\\n\\n[\\xa0\\n2\\n3...6\\n\\n\\xa0]', 'https://bbs.hupu.com/24553032.html', '幸运叶子体育', 'https://my.hupu.com/242729812037720', '2018-11-29', '112', '5414', '蝉噪林逾静', datetime.datetime(2018, 11, 29, 18, 13)]\n",
      "['网赌让我一下午欠了三万网贷，我会谨记自己现在狼狈的模样，远离赌博\\n\\n\\xa0\\n\\n[\\xa0\\n2\\n3...7\\n\\n\\xa0]', 'https://bbs.hupu.com/24554464.html', '四娃救爷爷', 'https://my.hupu.com/59783551435937', '2018-11-29', '130', '19966', '银77', datetime.datetime(2018, 11, 29, 18, 13)]\n",
      "['现在的吴悠应该处于人生的巅峰状态，祝他越来越好\\n\\n\\xa0\\n\\n[\\xa0\\n2\\n\\n\\xa0]', 'https://bbs.hupu.com/24551205.html', '顶级男神', 'https://my.hupu.com/20694473681250', '2018-11-29', '38', '8172', '传统型4000万先生', datetime.datetime(2018, 11, 29, 18, 13)]\n",
      "['关于传球', 'https://bbs.hupu.com/24555131.html', '燕歌起', 'https://my.hupu.com/254615604546704', '2018-11-29', '0', '0', '燕歌起', datetime.datetime(2018, 11, 29, 18, 13)]\n",
      "['一晚就能治疗感冒，身为医生给大家一些建议\\n\\n\\xa0\\n\\n[\\xa0\\n2\\n3\\n4\\n5\\n\\n\\xa0]', 'https://bbs.hupu.com/24553671.html', 'HealMomtH', 'https://my.hupu.com/281360417388606', '2018-11-29', '84', '9116', '小麦和他的理想国', datetime.datetime(2018, 11, 29, 18, 13)]\n",
      "['[家装房产]女朋友的父母被装修公司忽悠签了合同，被骗了3万多定金\\n\\n\\xa0\\n\\n[\\xa0\\n2\\n3\\n4\\n5\\n\\n\\xa0]', 'https://bbs.hupu.com/4836', '划船重来不用浆', 'https://my.hupu.com/230444686049988', '2018-11-29', '83', '27619', '齐柏林铁匠', datetime.datetime(2018, 11, 29, 18, 13)]\n",
      "['[车友交流]刚提了一辆蔚来es8，大家有什么想问的？\\n\\n\\xa0\\n\\n[\\xa0\\n2\\n3...23\\n\\n\\xa0]', 'https://bbs.hupu.com/cars', 'Antwan君', 'https://my.hupu.com/174298301323676', '2018-11-26', '442', '110808', 'Antwan君', datetime.datetime(2018, 11, 29, 18, 13)]\n",
      "['惊！凤凰遭史诗加强！', 'https://bbs.hupu.com/24554947.html', '2promise3', 'https://my.hupu.com/243880590905703', '2018-11-29', '11', '1611', '袋鼠SAMA', datetime.datetime(2018, 11, 29, 18, 13)]\n",
      "['对于婚前体检，你怎么看？\\n\\n\\xa0\\n\\n[\\xa0\\n2\\n3\\n4\\n\\n\\xa0]', 'https://bbs.hupu.com/24554444.html', '日月大神', 'https://my.hupu.com/200049430216410', '2018-11-29', '61', '8156', '屁屁喵', datetime.datetime(2018, 11, 29, 18, 13)]\n",
      "['吴亦凡当年这首《时间煮雨》是什么水平\\n\\n\\xa0\\n\\n[\\xa0\\n2\\n3...20\\n\\n\\xa0]', 'https://bbs.hupu.com/24550146.html', '顾烧Shaun', 'https://my.hupu.com/151314776554632', '2018-11-29', '399', '193675', '黑色柚子', datetime.datetime(2018, 11, 29, 18, 13)]\n",
      "['刚看到的一张图 这种舔狗是什么水平', 'https://bbs.hupu.com/24554269.html', 'txyswag', 'https://my.hupu.com/192446846971447', '2018-11-29', '13', '1568', '吃完麦辣肚子疼', datetime.datetime(2018, 11, 29, 18, 13)]\n",
      "['[影视区]华仔这波和蒙嘉慧的公交车邂逅什么水平\\n\\n\\xa0\\n\\n[\\xa0\\n2\\n3\\n\\n\\xa0]', 'https://bbs.hupu.com/ent', '哈哈哈虫合虫合', 'https://my.hupu.com/2864555737829', '2018-11-29', '42', '37183', 'mangocsr', datetime.datetime(2018, 11, 29, 18, 13)]\n",
      "['祁同伟如果当上了副省级，那后边应该咋样？', 'https://bbs.hupu.com/24554971.html', '半梦长久', 'https://my.hupu.com/54319871578570', '2018-11-29', '4', '484', '青龙烂', datetime.datetime(2018, 11, 29, 18, 13)]\n",
      "['客观的说，薛之谦的业务能力是什么水平？\\n\\n\\xa0\\n\\n[\\xa0\\n2\\n3...12\\n\\n\\xa0]', 'https://bbs.hupu.com/24547800.html', '她有些烫', 'https://my.hupu.com/39807346887370', '2018-11-29', '233', '41282', 'monster科', datetime.datetime(2018, 11, 29, 18, 13)]\n",
      "['中餐的国际地位为何没有日料高？', 'https://bbs.hupu.com/24555001.html', 'M弘哥哥', 'https://my.hupu.com/51372030565046', '2018-11-29', '15', '678', 'wwstarss', datetime.datetime(2018, 11, 29, 18, 13)]\n",
      "['[路人王专区]我们又来送福利啦！中日对抗为路人王加油，每天送一双实战球鞋！\\n\\n\\xa0\\n\\n[\\xa0\\n2\\n3...154\\n\\n\\xa0]', 'https://bbs.hupu.com/lurenwang', '路人王', 'https://my.hupu.com/100375168899168', '2018-11-28', '3068', '97000', '非洲非洲小白脸', datetime.datetime(2018, 11, 29, 18, 13)]\n",
      "['[路人王专区]8位种子选手及小组赛分组公布，冠军排位赛你更看好谁？', 'https://bbs.hupu.com/lurenwang', '路人王', 'https://my.hupu.com/100375168899168', '2018-11-29', '3', '2308', 'liuBBS', datetime.datetime(2018, 11, 29, 18, 13)]\n",
      "['朋友圈小姐姐发的，正在考虑要不要把她删了\\n\\n\\xa0\\n\\n[\\xa0\\n2\\n3...14\\n\\n\\xa0]', 'https://bbs.hupu.com/24545132.html', '梅西赛后表示', 'https://my.hupu.com/208129802616040', '2018-11-28', '275', '498527', '功波龟和饭悟栗', datetime.datetime(2018, 11, 29, 18, 13)]\n",
      "['[影视区]刘嘉玲26年前旧照像极佘诗曼，如今两人颜值下跌全是鼻孔惹的祸 zt', 'https://bbs.hupu.com/ent', '娱乐八卦阵', 'https://my.hupu.com/124819225349882', '2018-11-29', '4', '328', '火箭赛程好', datetime.datetime(2018, 11, 29, 18, 13)]\n",
      "['蒋劲夫接下来该如何洗地  才能出狱后继续捞钱？\\n\\n\\xa0\\n\\n[\\xa0\\n2\\n\\n\\xa0]', 'https://bbs.hupu.com/24554831.html', '见浦直人', 'https://my.hupu.com/261703347206075', '2018-11-29', '27', '3165', '我是韦德的朋友', datetime.datetime(2018, 11, 29, 18, 13)]\n",
      "['广大温暖的男jrs，在你们挫折沮丧的时候，你们最希望女票能做什么？', 'https://bbs.hupu.com/24554959.html', '七友1291', 'https://my.hupu.com/206741110927536', '2018-11-29', '2', '53', '七友1291', datetime.datetime(2018, 11, 29, 18, 13)]\n",
      "['第二次坐飞机，大家能不能教教我如何装成一个经常出差的商业人士hhh\\n\\n\\xa0\\n\\n[\\xa0\\n2\\n3\\n4\\n5\\n\\n\\xa0]', 'https://bbs.hupu.com/24554404.html', 'shashayy', 'https://my.hupu.com/131731475478024', '2018-11-29', '98', '16674', 'Yoghurt嘻嘻', datetime.datetime(2018, 11, 29, 18, 13)]\n",
      "['买不起1000万的法拉利，他花200万山寨了一辆 zt\\n\\n\\xa0\\n\\n[\\xa0\\n2\\n3...8\\n\\n\\xa0]', 'https://bbs.hupu.com/24550361.html', '生活资讯频道', 'https://my.hupu.com/132878375295650', '2018-11-29', '152', '175101', 'runaway_o', datetime.datetime(2018, 11, 29, 18, 13)]\n",
      "['这种姑娘算不算暴露？我很喜欢这种穿着的。\\n\\n\\xa0\\n\\n[\\xa0\\n2\\n\\n\\xa0]', 'https://bbs.hupu.com/24554843.html', '青灯丶行者', 'https://my.hupu.com/238775332831962', '2018-11-29', '31', '8154', '搞几个', datetime.datetime(2018, 11, 29, 18, 13)]\n",
      "['和陈羽凡同居的何某某（何时珍）是我朋友的大学同班同学什么水平？\\n\\n\\xa0\\n\\n[\\xa0\\n2\\n\\n\\xa0]', 'https://bbs.hupu.com/24554347.html', '花花是个公子', 'https://my.hupu.com/143051588904625', '2018-11-29', '35', '17494', 'babylow', datetime.datetime(2018, 11, 29, 18, 13)]\n",
      "['买了司法拍卖的房屋，原屋主赖着不走，买家起诉要求老赖搬离却败诉\\n\\n\\xa0\\n\\n[\\xa0\\n2\\n3\\n\\n\\xa0]', 'https://bbs.hupu.com/24552788.html', 'Angel按住啦掰逼', 'https://my.hupu.com/15523091608377', '2018-11-29', '45', '6704', '凯撒还是天天', datetime.datetime(2018, 11, 29, 18, 13)]\n",
      "['一句话让别人猜你玩的是什么游戏\\n\\n\\xa0\\n\\n[\\xa0\\n2\\n3...10\\n\\n\\xa0]', 'https://bbs.hupu.com/24550933.html', '社会肥宅阿雷哥', 'https://my.hupu.com/224378136667726', '2018-11-29', '184', '11913', 'Chunngai', datetime.datetime(2018, 11, 29, 18, 13)]\n",
      "['258人违规、21人永久禁赛！深圳南山半马大量参赛者作弊被罚 zt\\n\\n\\xa0\\n\\n[\\xa0\\n2\\n\\n\\xa0]', 'https://bbs.hupu.com/24554802.html', '每日热点新闻', 'https://my.hupu.com/43434294479022', '2018-11-29', '35', '20841', '超哥是泡泡', datetime.datetime(2018, 11, 29, 18, 13)]\n",
      "['大鹏乔杉按摩，这个腿！\\n\\n\\xa0\\n\\n[\\xa0\\n2\\n3...7\\n\\n\\xa0]', 'https://bbs.hupu.com/24553662.html', '街球style', 'https://my.hupu.com/154665693699407', '2018-11-29', '128', '142095', '孟菲斯托16', datetime.datetime(2018, 11, 29, 18, 13)]\n",
      "['成都大叔守4平小屋，一个人做锅盔32年，每天做700个全卖光  zt\\n\\n\\xa0\\n\\n[\\xa0\\n2\\n3...10\\n\\n\\xa0]', 'https://bbs.hupu.com/24540827.html', '生活资讯频道', 'https://my.hupu.com/132878375295650', '2018-11-28', '199', '132256', '神秘无感', datetime.datetime(2018, 11, 29, 18, 13)]\n",
      "['陈羽凡出事前天晚上海泉同框，大家怎么看？zt\\n\\n\\n\\xa0\\n\\n[\\xa0\\n2\\n3...6\\n\\n\\xa0]', 'https://bbs.hupu.com/24554463.html', '后撤步空心命中', 'https://my.hupu.com/82437628913401', '2018-11-29', '112', '89260', '永爱三号那个骚韦', datetime.datetime(2018, 11, 29, 18, 13)]\n",
      "['以后交流不犯愁，汉语正式加入“俄罗斯高考”！zt\\n\\n\\xa0\\n\\n[\\xa0\\n2\\n3...14\\n\\n\\xa0]', 'https://bbs.hupu.com/24551944.html', '锅包肉小李', 'https://my.hupu.com/131348616311400', '2018-11-29', '272', '141512', '真丶控卫格里芬', datetime.datetime(2018, 11, 29, 18, 13)]\n",
      "['借钱相关', 'https://bbs.hupu.com/24555095.html', '怼天怼地怼空气儿', 'https://my.hupu.com/208336875701061', '2018-11-29', '3', '49', 'yfyann', datetime.datetime(2018, 11, 29, 18, 13)]\n",
      "['岳不群翻唱的这首热狗代表作  差不多先生  什么水平？zt\\n\\n\\xa0\\n\\n[\\xa0\\n2\\n3...9\\n\\n\\xa0]', 'https://bbs.hupu.com/24553934.html', '凯里欧文表示如下', 'https://my.hupu.com/111826847350184', '2018-11-29', '173', '78532', '为了十八1', datetime.datetime(2018, 11, 29, 18, 13)]\n",
      "['女朋友觉得这种视频的内容是正确的  心累\\n\\n\\xa0\\n\\n[\\xa0\\n2\\n3...9\\n\\n\\xa0]', 'https://bbs.hupu.com/24550579.html', '直来直往好了', 'https://my.hupu.com/89103507179616', '2018-11-29', '175', '60260', '扎古怪', datetime.datetime(2018, 11, 29, 18, 13)]\n",
      "['微博上看到的，这个真的是合法能买到的最强防身武器吗？\\n\\n\\xa0\\n\\n[\\xa0\\n2\\n3...17\\n\\n\\xa0]', 'https://bbs.hupu.com/24548845.html', 'Clgrain38', 'https://my.hupu.com/90875127129702', '2018-11-29', '332', '455202', '果果球王', datetime.datetime(2018, 11, 29, 18, 13)]\n",
      "['要去指导女朋友的学院女篮球队 jrs们帮帮忙啊！\\n\\n\\xa0\\n\\n[\\xa0\\n2\\n3\\n4\\n\\n\\xa0]', 'https://bbs.hupu.com/24554114.html', 'xj完美弧线', 'https://my.hupu.com/79986938257269', '2018-11-29', '72', '4450', '嘉定吴彦祖他爸', datetime.datetime(2018, 11, 29, 18, 13)]\n",
      "['美娜在哈登面前大秀好身材, 哈登的眼神亮了\\n\\n\\xa0\\n\\n[\\xa0\\n2\\n3\\n4\\n\\n\\xa0]', 'https://bbs.hupu.com/24551459.html', '咸少布鲁克', 'https://my.hupu.com/156549750335714', '2018-11-29', '70', '93960', 'zzh34', datetime.datetime(2018, 11, 29, 18, 13)]\n",
      "['崴脚没消肿能走路么', 'https://bbs.hupu.com/24545339.html', '哐哐哐铁含量', 'https://my.hupu.com/164268357822236', '2018-11-28', '12', '506', '极限轮子', datetime.datetime(2018, 11, 29, 18, 13)]\n",
      "['其实我一直很好奇街上对纹身就好像杀人一样的厌恶\\n\\n\\xa0\\n\\n[\\xa0\\n2\\n3...16\\n\\n\\xa0]', 'https://bbs.hupu.com/24549571.html', '我皇必然君临天下', 'https://my.hupu.com/20448928116873', '2018-11-29', '312', '82806', 'EddiChoy', datetime.datetime(2018, 11, 29, 18, 13)]\n",
      "['孙红雷:别以为你是老大  杀你  还得你出钱', 'https://bbs.hupu.com/24555088.html', '耿浩在里面耿浩在里面', 'https://my.hupu.com/3821804325605', '2018-11-29', '1', '257', 'Bossqiao', datetime.datetime(2018, 11, 29, 18, 13)]\n",
      "['D&G事件后 我们是否能顶住不当舔狗？', 'https://bbs.hupu.com/24555076.html', 'LebronXIII', 'https://my.hupu.com/91224593376539', '2018-11-29', '4', '390', 'wrguo', datetime.datetime(2018, 11, 29, 18, 13)]\n",
      "['新婚少妇唐嫣这一波性感露肩短裙+过膝长靴诱惑什么水平？\\n\\n\\xa0\\n\\n[\\xa0\\n2\\n3...12\\n\\n\\xa0]', 'https://bbs.hupu.com/24551807.html', 'kstdlds', 'https://my.hupu.com/184756325656505', '2018-11-29', '225', '251219', '强插章泽天', datetime.datetime(2018, 11, 29, 18, 13)]\n",
      "['给前男友借了三千 我也不知道自己怎么想的\\n\\n\\xa0\\n\\n[\\xa0\\n2\\n3...7\\n\\n\\xa0]', 'https://bbs.hupu.com/24546612.html', 'sarabald', 'https://my.hupu.com/68649339487633', '2018-11-28', '130', '22452', 'sarabald', datetime.datetime(2018, 11, 29, 18, 13)]\n",
      "['板娘的死亡骆驼    我不是音痴！\\n\\n\\xa0\\n\\n[\\xa0\\n2\\n\\n\\xa0]', 'https://bbs.hupu.com/24554346.html', '空城空心空等待', 'https://my.hupu.com/16502747078701', '2018-11-29', '38', '4596', 'In1tial86D', datetime.datetime(2018, 11, 29, 18, 13)]\n",
      "['我们学校新生杯小组赛十佳球 什么水平？\\n\\n\\xa0\\n\\n[\\xa0\\n2\\n3\\n\\n\\xa0]', 'https://bbs.hupu.com/24548106.html', 'CARROVIN', 'https://my.hupu.com/173690247862634', '2018-11-29', '52', '11567', 'CARROVIN', datetime.datetime(2018, 11, 29, 18, 13)]\n",
      "['大家好，你们那里有好吃的面吗？\\n\\n\\xa0\\n\\n[\\xa0\\n2\\n3...11\\n\\n\\xa0]', 'https://bbs.hupu.com/24551867.html', '梦想实现家', 'https://my.hupu.com/121410402031557', '2018-11-29', '209', '14761', '詹皇亲完骚韦说', datetime.datetime(2018, 11, 29, 18, 13)]\n",
      "['齐达内围观杭州老太搓麻将，感慨太玄妙zt', 'https://bbs.hupu.com/24554980.html', 'shisonglph', 'https://my.hupu.com/12119043273832', '2018-11-29', '6', '701', '七星冷刀', datetime.datetime(2018, 11, 29, 18, 13)]\n",
      "['女朋友想看我的相册，我没让她看，她生气了，怎么哄啊\\n\\n\\xa0\\n\\n[\\xa0\\n2\\n\\n\\xa0]', 'https://bbs.hupu.com/24554134.html', '寂寞如老酒', 'https://my.hupu.com/211116898538352', '2018-11-29', '23', '3681', '徐景依', datetime.datetime(2018, 11, 29, 18, 13)]\n",
      "['今天来KFC 体验了双份的孤独', 'https://bbs.hupu.com/24555030.html', 'ai佛僧', 'https://my.hupu.com/89278080068837', '2018-11-29', '10', '1800', 'ai佛僧', datetime.datetime(2018, 11, 29, 18, 13)]\n",
      "['电影《保持通话》片段，JRs们觉得王祖蓝演技如何？\\n\\n\\xa0\\n\\n[\\xa0\\n2\\n3...7\\n\\n\\xa0]', 'https://bbs.hupu.com/24549371.html', '老徐侃球', 'https://my.hupu.com/132225419087847', '2018-11-29', '120', '68519', '晓晚丶', datetime.datetime(2018, 11, 29, 18, 13)]\n",
      "['来来来，你们要的羽泉我是歌手的《大地》，纪念羽凡的好嗓子\\n\\n\\xa0\\n\\n[\\xa0\\n2\\n3...16\\n\\n\\xa0]', 'https://bbs.hupu.com/24551472.html', 'Owen1203', 'https://my.hupu.com/49249505259994', '2018-11-29', '314', '128536', 'xcy葱花', datetime.datetime(2018, 11, 29, 18, 13)]\n",
      "['我的妈妈是越南人。zt\\n\\n\\xa0\\n\\n[\\xa0\\n2\\n3...29\\n\\n\\xa0]', 'https://bbs.hupu.com/24549289.html', '我叫小肥乐', 'https://my.hupu.com/73108850631531', '2018-11-29', '564', '340508', '我的小葵', datetime.datetime(2018, 11, 29, 18, 13)]\n"
     ]
    }
   ],
   "source": [
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "import datetime\n",
    "\n",
    "# 获取页面\n",
    "def get_page(link):\n",
    "    headers = {'User-Agent' : 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6'} \n",
    "    r = requests.get(link, headers = headers)\n",
    "    html = r.content  #使用r.content解封装\n",
    "    html = html.decode('utf-8')  #由UTF-8解码为unicode\n",
    "    soup = BeautifulSoup(html, 'lxml')\n",
    "    return soup\n",
    "\n",
    "# 解析网页\n",
    "def get_data(post_list):\n",
    "    data_list =[]\n",
    "    for post in post_list:\n",
    "        title = post.find('div',class_='titlelink box').text.strip()\n",
    "        post_link = post.find('div',class_='titlelink box').a['href']\n",
    "        post_link = \"https://bbs.hupu.com\" + post_link \n",
    "\n",
    "        author = post.find('div',class_='author box').a.text.strip()\n",
    "        author_page = post.find('div',class_='author box').a['href']\n",
    "        start_date = post.find('div',class_='author box').contents[5].text.strip()\n",
    "\n",
    "        reply_view = post.find('span',class_='ansour box').text.strip()\n",
    "        reply = reply_view.split('/')[0].strip()\n",
    "        view = reply_view.split('/')[1].strip()\n",
    "\n",
    "        reply_time = post.find('div',class_='endreply box').a.text.strip()\n",
    "        last_reply = post.find('div',class_='endreply box').span.text.strip()\n",
    "        if ':' in reply_time: #时间是11:27\n",
    "            date_time = str(datetime.date.today()) + ' ' + reply_time\n",
    "            date_time = datetime.datetime.strptime(date_time, '%Y-%m-%d %H:%M')\n",
    "        elif reply_time.find(\"-\") == 4: #时间是2017-02-27\n",
    "            date_time = datetime.datetime.strptime(reply_time, '%Y-%m-%d').date()\n",
    "        else: #时间是11-27\n",
    "            date_time = datetime.datetime.strptime('2018-' + reply_time, '%Y-%m-%d').date()\n",
    "        data_list.append([title, post_link, author, author_page, start_date, reply, view, last_reply, date_time])\n",
    "    return data_list\n",
    "\n",
    "link = \"https://bbs.hupu.com/bxj\"\n",
    "soup = get_page(link)\n",
    "post_all= soup.find('ul', class_=\"for-list\")\n",
    "post_list = post_all.find_all('li')\n",
    "data_list = get_data(post_list)\n",
    "for each in data_list:\n",
    "    print (each)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-11-29T10:14:46.977599Z",
     "start_time": "2018-11-29T10:14:46.973590Z"
    }
   },
   "outputs": [],
   "source": [
    "from pymongo import MongoClient\n",
    "class MongoAPI(object):\n",
    "    def __init__(self, db_ip, db_port, db_name, table_name):\n",
    "        self.db_ip = db_ip\n",
    "        self.db_port = db_port\n",
    "        self.db_name = db_name\n",
    "        self.table_name = table_name\n",
    "        self.conn = MongoClient(host=self.db_ip, port=self.db_port)\n",
    "        self.db = self.conn[self.db_name]\n",
    "        self.table = self.db[self.table_name]\n",
    "    def get_one(self, query):\n",
    "        return self.table.find_one(query, projection={\"_id\": False})\n",
    "    def get_all(self, query):\n",
    "        return self.table.find(query)\n",
    "    def add(self, kv_dict):\n",
    "        return self.table.insert(kv_dict)\n",
    "    def delete(self, query):\n",
    "        return self.table.delete_many(query)\n",
    "    def check_exist(self, query):\n",
    "        ret = self.table.find_one(query)\n",
    "        return ret != None\n",
    "    # 如果没有会新建\n",
    "    def update(self, query, kv_dict):\n",
    "            self.table.update_one(query,{\n",
    "              '$set': kv_dict\n",
    "            }, upsert=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-11-29T10:15:02.622733Z",
     "start_time": "2018-11-29T10:15:02.536533Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\santostang\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\ipykernel_launcher.py:16: DeprecationWarning: insert is deprecated. Use insert_one or insert_many instead.\n",
      "  app.launch_new_instance()\n"
     ]
    }
   ],
   "source": [
    "hupu_post = MongoAPI(\"localhost\",  27017,  \"hupu\", \"post\")\n",
    "for each in data_list:\n",
    "    hupu_post.add({\"title\": each[0], \n",
    "                \"post_link\": each[1],\n",
    "               \"author\": each[2],\n",
    "               \"author_page\": each[3],\n",
    "               \"start_date\": str(each[4]),\n",
    "               \"reply\": each[5],\n",
    "               \"view\": each[6],\n",
    "               \"last_reply\": each[7],\n",
    "               \"last_reply_time\": str(each[8])})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-11-29T11:25:23.857133Z",
     "start_time": "2018-11-29T11:24:48.247568Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "第 1 页获取完成，休息3秒\n",
      "第 2 页获取完成，休息3秒\n",
      "第 3 页获取完成，休息3秒\n",
      "第 4 页获取完成，休息3秒\n",
      "第 5 页获取完成，休息3秒\n",
      "第 6 页获取完成，休息3秒\n",
      "第 7 页获取完成，休息3秒\n",
      "第 8 页获取完成，休息3秒\n",
      "第 9 页获取完成，休息3秒\n",
      "第 10 页获取完成，休息3秒\n"
     ]
    },
    {
     "ename": "AttributeError",
     "evalue": "'NoneType' object has no attribute 'find_all'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-18-dcddeebadc4a>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     11\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     12\u001b[0m     \u001b[0mpost_all\u001b[0m\u001b[1;33m=\u001b[0m \u001b[0msoup\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfind\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'ul'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mclass_\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m\"for-list\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 13\u001b[1;33m     \u001b[0mpost_list\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mpost_all\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfind_all\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'li'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     14\u001b[0m     \u001b[0mdata_list\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mget_data\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mpost_list\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     15\u001b[0m     \u001b[1;32mfor\u001b[0m \u001b[0meach\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mdata_list\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mAttributeError\u001b[0m: 'NoneType' object has no attribute 'find_all'"
     ]
    }
   ],
   "source": [
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "import datetime\n",
    "from pymongo import MongoClient\n",
    "import time\n",
    "\n",
    "hupu_post = MongoAPI(\"localhost\",  27017,  \"hupu\", \"post\")\n",
    "for i in range(1,100):\n",
    "    link = \"https://bbs.hupu.com/bxj-\" + str(i)\n",
    "    soup = get_page(link)\n",
    "    \n",
    "    post_all= soup.find('ul', class_=\"for-list\")\n",
    "    post_list = post_all.find_all('li')\n",
    "    data_list = get_data(post_list)\n",
    "    for each in data_list:\n",
    "        hupu_post.update({\"post_link\": each[1]},{\"title\": each[0], \n",
    "                    \"post_link\": each[1],\n",
    "                   \"author\": each[2],\n",
    "                   \"author_page\": each[3],\n",
    "                   \"start_date\": str(each[4]),\n",
    "                   \"reply\": each[5],\n",
    "                   \"view\": each[6],\n",
    "                   \"last_reply\": each[7],\n",
    "                   \"last_reply_time\": str(each[8])})\n",
    "    time.sleep(3)\n",
    "    print ('第', i ,'页获取完成，休息3秒')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {
    "height": "12px",
    "width": "252px"
   },
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": "block",
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
